{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "import transformers\n",
    "import matplotlib.pyplot as plt\n",
    "from random import seed\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "import torch.utils.data as data\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.warmup预热学习率\n",
    "### What is warmup?\n",
    "* 它在训练开始的时候先选择使用一个较小的学习率，\n",
    "* 训练了一些epoches或者steps,再修改为预先设置的学习率来进行训练  \n",
    "\n",
    "### Why use it?\n",
    "* 刚开始训练时,模型的权重(weights)是随机初始化的\n",
    "    * 此时若选择一个较大的学习率,可能带来模型的不稳定(振荡)\n",
    "* 选择Warmup预热学习率的方式\n",
    "    * 可以使得开始训练的几个epoches或者一些steps内学习率较小\n",
    "    * 在预热的小学习率下，模型可以慢慢趋于稳定,等模型相对稳定后再选择预先设置的学习率进行训练,使得模型收敛速度变得更快，模型效果更佳。\n",
    "\n",
    "\n",
    "### the improvement of warmup\n",
    "* 它的不足之处在于从一个很小的学习率一下变为比较大的学习率可能会导致训练误差突然增大。\n",
    "* 于是18年Facebook提出了gradual warmup来解决这个问题，\n",
    "    * 即从最初的小学习率开始，每个step增大一点点，直到达到最初设置的比较大的学习率时"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_steps:100.000--warmup_steps:1500.000--learning_rate:0.007\n",
      "train_steps:200.000--warmup_steps:1500.000--learning_rate:0.013\n",
      "train_steps:300.000--warmup_steps:1500.000--learning_rate:0.020\n",
      "train_steps:400.000--warmup_steps:1500.000--learning_rate:0.027\n",
      "train_steps:500.000--warmup_steps:1500.000--learning_rate:0.033\n",
      "train_steps:600.000--warmup_steps:1500.000--learning_rate:0.040\n",
      "train_steps:700.000--warmup_steps:1500.000--learning_rate:0.047\n",
      "train_steps:800.000--warmup_steps:1500.000--learning_rate:0.053\n",
      "train_steps:900.000--warmup_steps:1500.000--learning_rate:0.060\n",
      "train_steps:1000.000--warmup_steps:1500.000--learning_rate:0.067\n",
      "train_steps:1100.000--warmup_steps:1500.000--learning_rate:0.073\n",
      "train_steps:1200.000--warmup_steps:1500.000--learning_rate:0.080\n",
      "train_steps:1300.000--warmup_steps:1500.000--learning_rate:0.087\n",
      "train_steps:1400.000--warmup_steps:1500.000--learning_rate:0.093\n",
      "train_steps:1500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:1600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:1700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:1800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:1900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:2000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:2100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:2200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:2300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:2400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:2500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:2600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:2700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:2800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:2900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:3000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:3100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:3200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:3300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:3400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:3500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:3600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:3700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:3800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:3900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:4000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:4100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:4200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:4300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:4400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:4500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:4600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:4700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:4800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:4900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:5000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:5100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:5200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:5300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:5400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:5500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:5600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:5700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:5800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:5900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:6000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:6100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:6200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:6300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:6400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:6500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:6600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:6700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:6800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:6900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:7000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:7100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:7200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:7300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:7400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:7500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:7600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:7700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:7800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:7900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:8000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:8100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:8200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:8300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:8400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:8500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:8600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:8700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:8800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:8900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:9000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:9100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:9200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:9300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:9400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:9500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:9600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:9700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:9800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:9900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:10000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:10100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:10200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:10300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:10400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:10500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:10600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:10700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:10800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:10900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:11000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:11100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:11200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:11300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:11400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:11500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:11600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:11700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:11800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:11900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:12000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:12100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:12200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:12300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:12400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:12500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:12600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:12700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:12800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:12900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:13000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:13100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:13200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:13300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:13400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:13500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:13600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:13700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:13800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:13900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:14000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:14100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:14200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:14300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:14400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:14500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:14600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:14700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:14800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:14900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:15000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:15100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:15200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:15300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:15400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:15500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:15600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:15700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:15800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:15900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:16000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:16100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:16200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:16300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:16400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:16500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:16600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:16700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:16800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:16900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:17000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:17100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:17200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:17300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:17400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:17500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:17600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:17700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:17800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:17900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:18000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:18100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:18200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:18300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:18400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:18500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:18600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:18700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:18800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:18900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:19000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:19100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:19200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:19300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:19400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:19500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:19600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:19700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:19800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:19900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:20000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:20100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:20200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:20300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:20400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:20500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:20600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:20700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:20800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:20900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:21000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:21100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:21200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:21300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:21400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:21500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:21600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:21700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:21800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:21900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:22000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:22100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:22200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:22300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:22400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:22500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:22600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:22700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:22800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:22900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:23000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:23100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:23200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:23300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:23400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:23500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:23600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:23700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:23800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:23900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:24000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:24100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:24200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:24300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:24400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:24500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:24600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:24700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:24800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:24900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:25000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:25100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:25200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:25300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:25400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:25500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:25600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:25700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:25800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:25900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:26000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:26100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:26200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:26300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:26400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:26500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:26600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:26700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:26800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:26900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:27000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:27100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:27200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:27300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:27400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:27500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:27600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:27700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:27800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:27900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:28000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:28100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:28200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:28300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:28400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:28500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:28600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:28700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:28800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:28900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:29000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:29100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:29200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:29300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:29400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:29500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:29600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:29700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:29800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:29900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:30000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:30100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:30200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:30300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:30400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:30500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:30600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:30700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:30800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:30900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:31000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:31100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:31200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:31300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:31400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:31500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:31600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:31700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:31800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:31900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:32000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:32100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:32200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:32300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:32400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:32500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:32600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:32700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:32800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:32900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:33000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:33100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:33200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:33300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:33400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:33500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:33600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:33700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:33800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:33900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:34000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:34100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:34200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:34300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:34400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:34500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:34600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:34700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:34800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:34900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:35000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:35100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:35200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:35300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:35400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:35500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:35600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:35700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:35800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:35900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:36000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:36100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:36200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:36300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:36400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:36500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:36600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:36700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:36800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:36900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:37000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:37100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:37200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:37300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:37400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:37500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:37600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:37700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:37800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:37900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:38000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:38100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:38200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:38300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:38400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:38500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:38600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:38700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:38800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:38900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:39000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:39100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:39200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:39300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:39400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:39500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:39600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:39700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:39800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:39900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:40000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:40100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:40200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:40300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:40400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:40500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:40600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:40700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:40800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:40900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:41000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:41100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:41200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:41300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:41400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:41500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:41600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:41700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:41800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:41900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:42000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:42100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:42200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:42300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:42400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:42500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:42600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:42700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:42800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:42900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:43000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:43100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:43200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:43300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:43400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:43500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:43600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:43700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:43800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:43900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:44000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:44100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:44200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:44300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:44400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:44500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:44600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:44700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:44800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:44900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:45000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:45100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:45200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:45300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:45400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:45500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:45600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:45700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:45800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:45900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:46000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:46100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:46200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:46300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:46400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:46500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:46600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:46700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:46800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:46900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:47000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:47100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:47200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:47300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:47400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:47500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:47600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:47700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:47800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:47900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:48000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:48100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:48200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:48300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:48400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:48500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:48600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:48700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:48800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:48900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:49000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:49100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:49200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:49300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:49400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:49500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:49600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:49700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:49800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:49900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:50000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:50100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:50200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:50300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:50400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:50500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:50600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:50700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:50800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:50900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:51000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:51100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:51200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:51300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:51400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:51500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:51600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:51700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:51800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:51900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:52000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:52100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:52200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:52300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:52400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:52500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:52600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:52700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:52800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:52900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:53000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:53100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:53200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:53300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:53400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:53500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:53600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:53700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:53800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:53900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:54000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:54100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:54200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:54300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:54400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:54500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:54600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:54700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:54800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:54900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:55000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:55100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:55200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:55300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:55400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:55500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:55600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:55700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:55800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:55900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:56000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:56100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:56200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:56300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:56400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:56500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:56600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:56700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:56800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:56900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:57000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:57100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:57200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:57300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:57400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:57500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:57600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:57700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:57800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:57900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:58000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:58100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:58200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:58300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:58400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:58500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:58600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:58700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:58800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:58900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:59000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:59100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:59200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:59300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:59400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:59500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:59600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:59700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:59800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:59900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:60000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:60100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:60200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:60300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:60400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:60500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:60600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:60700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:60800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:60900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:61000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:61100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:61200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:61300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:61400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:61500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:61600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:61700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:61800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:61900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:62000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:62100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:62200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:62300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:62400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:62500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:62600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:62700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:62800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:62900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:63000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:63100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:63200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:63300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:63400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:63500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:63600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:63700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:63800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:63900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:64000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:64100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:64200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:64300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:64400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:64500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:64600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:64700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:64800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:64900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:65000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:65100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:65200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:65300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:65400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:65500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:65600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:65700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:65800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:65900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:66000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:66100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:66200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:66300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:66400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:66500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:66600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:66700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:66800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:66900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:67000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:67100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:67200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:67300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:67400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:67500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:67600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:67700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:67800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:67900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:68000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:68100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:68200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:68300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:68400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:68500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:68600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:68700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:68800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:68900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:69000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:69100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:69200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:69300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:69400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:69500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:69600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:69700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:69800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:69900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:70000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:70100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:70200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:70300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:70400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:70500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:70600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:70700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:70800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:70900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:71000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:71100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:71200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:71300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:71400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:71500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:71600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:71700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:71800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:71900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:72000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:72100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:72200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:72300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:72400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:72500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:72600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:72700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:72800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:72900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:73000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:73100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:73200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:73300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:73400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:73500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:73600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:73700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:73800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:73900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:74000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:74100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:74200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:74300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:74400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:74500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:74600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:74700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:74800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:74900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:75000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:75100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:75200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:75300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:75400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:75500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:75600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:75700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:75800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:75900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:76000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:76100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:76200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:76300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:76400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:76500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:76600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:76700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:76800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:76900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:77000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:77100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:77200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:77300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:77400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:77500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:77600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:77700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:77800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:77900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:78000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:78100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:78200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:78300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:78400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:78500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:78600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:78700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:78800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:78900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:79000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:79100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:79200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:79300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:79400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:79500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:79600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:79700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:79800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:79900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:80000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:80100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:80200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:80300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:80400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:80500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:80600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:80700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:80800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:80900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:81000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:81100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:81200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:81300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:81400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:81500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:81600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:81700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:81800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:81900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:82000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:82100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:82200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:82300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:82400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:82500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:82600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:82700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:82800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:82900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:83000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:83100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:83200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:83300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:83400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:83500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:83600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:83700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:83800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:83900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:84000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:84100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:84200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:84300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:84400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:84500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:84600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:84700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:84800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:84900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:85000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:85100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:85200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:85300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:85400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:85500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:85600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:85700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:85800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:85900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:86000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:86100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:86200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:86300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:86400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:86500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:86600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:86700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:86800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:86900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:87000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:87100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:87200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:87300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:87400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:87500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:87600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:87700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:87800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:87900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:88000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:88100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:88200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:88300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:88400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:88500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:88600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:88700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:88800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:88900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:89000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:89100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:89200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:89300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:89400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:89500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:89600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:89700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:89800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:89900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:90000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:90100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:90200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:90300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:90400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:90500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:90600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:90700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:90800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:90900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:91000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:91100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:91200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:91300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:91400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:91500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:91600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:91700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:91800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:91900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:92000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:92100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:92200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:92300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:92400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:92500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:92600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:92700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:92800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:92900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:93000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:93100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:93200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:93300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:93400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:93500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:93600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:93700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:93800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:93900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:94000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:94100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:94200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:94300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:94400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:94500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:94600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:94700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:94800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:94900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:95000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:95100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:95200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:95300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:95400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:95500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:95600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:95700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:95800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:95900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:96000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:96100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:96200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:96300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:96400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:96500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:96600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:96700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:96800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:96900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:97000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:97100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:97200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:97300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:97400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:97500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:97600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:97700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:97800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:97900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:98000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:98100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:98200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:98300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:98400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:98500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:98600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:98700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:98800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:98900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:99000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:99100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:99200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:99300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:99400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:99500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:99600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:99700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:99800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:99900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:100000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:100100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:100200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:100300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:100400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:100500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:100600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:100700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:100800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:100900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:101000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:101100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:101200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:101300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:101400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:101500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:101600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:101700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:101800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:101900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:102000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:102100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:102200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:102300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:102400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:102500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:102600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:102700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:102800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:102900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:103000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:103100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:103200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:103300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:103400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:103500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:103600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:103700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:103800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:103900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:104000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:104100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:104200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:104300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:104400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:104500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:104600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:104700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:104800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:104900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:105000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:105100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:105200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:105300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:105400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:105500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:105600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:105700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:105800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:105900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:106000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:106100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:106200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:106300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:106400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:106500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:106600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:106700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:106800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:106900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:107000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:107100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:107200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:107300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:107400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:107500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:107600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:107700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:107800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:107900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:108000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:108100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:108200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:108300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:108400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:108500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:108600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:108700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:108800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:108900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:109000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:109100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:109200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:109300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:109400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:109500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:109600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:109700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:109800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:109900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:110000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:110100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:110200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:110300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:110400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:110500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:110600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:110700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:110800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:110900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:111000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:111100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:111200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:111300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:111400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:111500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:111600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:111700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:111800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:111900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:112000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:112100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:112200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:112300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:112400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:112500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:112600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:112700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:112800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:112900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:113000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:113100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:113200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:113300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:113400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:113500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:113600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:113700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:113800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:113900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:114000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:114100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:114200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:114300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:114400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:114500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:114600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:114700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:114800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:114900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:115000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:115100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:115200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:115300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:115400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:115500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:115600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:115700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:115800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:115900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:116000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:116100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:116200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:116300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:116400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:116500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:116600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:116700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:116800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:116900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:117000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:117100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:117200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:117300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:117400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:117500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:117600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:117700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:117800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:117900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:118000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:118100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:118200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:118300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:118400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:118500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:118600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:118700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:118800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:118900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:119000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:119100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:119200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:119300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:119400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:119500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:119600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:119700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:119800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:119900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:120000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:120100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:120200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:120300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:120400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:120500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:120600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:120700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:120800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:120900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:121000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:121100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:121200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:121300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:121400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:121500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:121600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:121700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:121800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:121900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:122000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:122100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:122200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:122300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:122400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:122500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:122600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:122700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:122800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:122900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:123000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:123100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:123200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:123300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:123400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:123500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:123600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:123700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:123800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:123900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:124000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:124100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:124200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:124300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:124400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:124500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:124600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:124700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:124800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:124900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:125000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:125100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:125200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:125300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:125400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:125500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:125600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:125700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:125800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:125900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:126000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:126100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:126200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:126300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:126400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:126500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:126600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:126700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:126800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:126900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:127000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:127100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:127200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:127300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:127400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:127500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:127600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:127700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:127800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:127900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:128000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:128100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:128200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:128300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:128400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:128500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:128600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:128700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:128800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:128900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:129000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:129100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:129200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:129300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:129400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:129500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:129600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:129700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:129800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:129900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:130000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:130100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:130200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:130300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:130400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:130500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:130600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:130700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:130800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:130900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:131000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:131100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:131200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:131300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:131400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:131500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:131600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:131700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:131800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:131900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:132000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:132100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:132200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:132300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:132400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:132500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:132600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:132700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:132800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:132900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:133000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:133100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:133200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:133300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:133400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:133500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:133600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:133700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:133800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:133900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:134000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:134100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:134200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:134300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:134400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:134500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:134600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:134700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:134800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:134900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:135000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:135100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:135200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:135300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:135400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:135500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:135600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:135700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:135800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:135900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:136000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:136100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:136200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:136300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:136400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:136500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:136600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:136700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:136800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:136900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:137000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:137100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:137200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:137300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:137400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:137500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:137600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:137700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:137800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:137900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:138000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:138100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:138200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:138300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:138400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:138500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:138600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:138700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:138800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:138900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:139000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:139100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:139200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:139300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:139400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:139500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:139600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:139700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:139800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:139900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:140000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:140100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:140200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:140300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:140400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:140500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:140600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:140700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:140800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:140900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:141000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:141100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:141200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:141300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:141400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:141500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:141600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:141700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:141800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:141900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:142000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:142100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:142200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:142300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:142400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:142500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:142600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:142700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:142800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:142900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:143000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:143100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:143200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:143300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:143400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:143500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:143600.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:143700.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:143800.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:143900.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:144000.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:144100.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:144200.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:144300.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:144400.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:144500.000--warmup_steps:1500.000--learning_rate:0.100\n",
      "train_steps:144600.000--warmup_steps:1500.000--learning_rate:0.098\n",
      "train_steps:144700.000--warmup_steps:1500.000--learning_rate:0.095\n",
      "train_steps:144800.000--warmup_steps:1500.000--learning_rate:0.093\n",
      "train_steps:144900.000--warmup_steps:1500.000--learning_rate:0.091\n",
      "train_steps:145000.000--warmup_steps:1500.000--learning_rate:0.089\n",
      "train_steps:145100.000--warmup_steps:1500.000--learning_rate:0.087\n",
      "train_steps:145200.000--warmup_steps:1500.000--learning_rate:0.085\n",
      "train_steps:145300.000--warmup_steps:1500.000--learning_rate:0.082\n",
      "train_steps:145400.000--warmup_steps:1500.000--learning_rate:0.080\n",
      "train_steps:145500.000--warmup_steps:1500.000--learning_rate:0.078\n",
      "train_steps:145600.000--warmup_steps:1500.000--learning_rate:0.076\n",
      "train_steps:145700.000--warmup_steps:1500.000--learning_rate:0.075\n",
      "train_steps:145800.000--warmup_steps:1500.000--learning_rate:0.073\n",
      "train_steps:145900.000--warmup_steps:1500.000--learning_rate:0.071\n",
      "train_steps:146000.000--warmup_steps:1500.000--learning_rate:0.069\n",
      "train_steps:146100.000--warmup_steps:1500.000--learning_rate:0.067\n",
      "train_steps:146200.000--warmup_steps:1500.000--learning_rate:0.065\n",
      "train_steps:146300.000--warmup_steps:1500.000--learning_rate:0.063\n",
      "train_steps:146400.000--warmup_steps:1500.000--learning_rate:0.062\n",
      "train_steps:146500.000--warmup_steps:1500.000--learning_rate:0.060\n",
      "train_steps:146600.000--warmup_steps:1500.000--learning_rate:0.058\n",
      "train_steps:146700.000--warmup_steps:1500.000--learning_rate:0.057\n",
      "train_steps:146800.000--warmup_steps:1500.000--learning_rate:0.055\n",
      "train_steps:146900.000--warmup_steps:1500.000--learning_rate:0.054\n",
      "train_steps:147000.000--warmup_steps:1500.000--learning_rate:0.052\n",
      "train_steps:147100.000--warmup_steps:1500.000--learning_rate:0.050\n",
      "train_steps:147200.000--warmup_steps:1500.000--learning_rate:0.049\n",
      "train_steps:147300.000--warmup_steps:1500.000--learning_rate:0.047\n",
      "train_steps:147400.000--warmup_steps:1500.000--learning_rate:0.046\n",
      "train_steps:147500.000--warmup_steps:1500.000--learning_rate:0.045\n",
      "train_steps:147600.000--warmup_steps:1500.000--learning_rate:0.043\n",
      "train_steps:147700.000--warmup_steps:1500.000--learning_rate:0.042\n",
      "train_steps:147800.000--warmup_steps:1500.000--learning_rate:0.041\n",
      "train_steps:147900.000--warmup_steps:1500.000--learning_rate:0.039\n",
      "train_steps:148000.000--warmup_steps:1500.000--learning_rate:0.038\n",
      "train_steps:148100.000--warmup_steps:1500.000--learning_rate:0.037\n",
      "train_steps:148200.000--warmup_steps:1500.000--learning_rate:0.036\n",
      "train_steps:148300.000--warmup_steps:1500.000--learning_rate:0.034\n",
      "train_steps:148400.000--warmup_steps:1500.000--learning_rate:0.033\n",
      "train_steps:148500.000--warmup_steps:1500.000--learning_rate:0.032\n",
      "train_steps:148600.000--warmup_steps:1500.000--learning_rate:0.031\n",
      "train_steps:148700.000--warmup_steps:1500.000--learning_rate:0.030\n",
      "train_steps:148800.000--warmup_steps:1500.000--learning_rate:0.029\n",
      "train_steps:148900.000--warmup_steps:1500.000--learning_rate:0.028\n",
      "train_steps:149000.000--warmup_steps:1500.000--learning_rate:0.027\n",
      "train_steps:149100.000--warmup_steps:1500.000--learning_rate:0.026\n",
      "train_steps:149200.000--warmup_steps:1500.000--learning_rate:0.025\n",
      "train_steps:149300.000--warmup_steps:1500.000--learning_rate:0.024\n",
      "train_steps:149400.000--warmup_steps:1500.000--learning_rate:0.023\n",
      "train_steps:149500.000--warmup_steps:1500.000--learning_rate:0.022\n",
      "train_steps:149600.000--warmup_steps:1500.000--learning_rate:0.022\n",
      "train_steps:149700.000--warmup_steps:1500.000--learning_rate:0.021\n",
      "train_steps:149800.000--warmup_steps:1500.000--learning_rate:0.020\n",
      "train_steps:149900.000--warmup_steps:1500.000--learning_rate:0.019\n",
      "train_steps:150000.000--warmup_steps:1500.000--learning_rate:0.018\n"
     ]
    }
   ],
   "source": [
    "warmup_steps = 1500\n",
    "init_lr = 0.1\n",
    "\n",
    "max_epoch = 150000\n",
    "\n",
    "# 训练150000个轮次\n",
    "for train_steps in range(max_epoch):\n",
    "    # 在前warmp_steps轮次进行预热\n",
    "    if warmup_steps and train_steps < warmup_steps:\n",
    "        warmup_percent_done = train_steps / warmup_steps\n",
    "        warmup_learning_rate = init_lr * warmup_percent_done  \n",
    "        #gradual warmup_lr\n",
    "        learning_rate = warmup_learning_rate\n",
    "    elif train_steps < 144500:\n",
    "    #learning_rate = np.sin(learning_rate)  \n",
    "    #预热学习率结束后,学习率呈sin衰减\n",
    "        learning_rate = learning_rate \n",
    "    #预热学习率结束后,学习率呈指数衰减(近似模拟指数衰减)\n",
    "    else:\n",
    "        learning_rate = learning_rate ** 1.0001\n",
    "    if (train_steps+1) % 100 == 0:\n",
    "             print(\"train_steps:%.3f--warmup_steps:%.3f--learning_rate:%.3f\" % (\n",
    "                 train_steps+1, warmup_steps, learning_rate))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.learning rate decay学习率衰减\n",
    "在训练模型的时候，通常会遇到这种情况：  \n",
    "<font color=\"red\">**我们平衡模型的训练速度和损失（loss）后选择了相对合适的学习率（learning rate），但是训练集的损失下降到一定的程度后就不在下降了，比如training loss一直在0.7和0.9之间来回震荡，不能进一步下降**</font>  \n",
    "* 遇到这种情况通常可以通过适当**降低学习率**（learning rate）来实现。\n",
    "* 但是，**降低学习率又会延长训练所需的时间**。 \n",
    "***  \n",
    "### 学习率衰减（learning rate decay） 就是一种可以平衡这两者之间矛盾的解决方案。\n",
    "* 学习率随着训练的进行逐渐衰减。\n",
    "    + 线性衰减。例如：每过5个epochs学习率减半。  \n",
    "    + 指数衰减。例如：随着迭代轮数的增加学习率自动发生衰减，每过5个epochs将学习率乘以0.9998。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.nn.Linear()设置网络中的全连接层（线性层）\n",
    "<font color=\"green\">**其每个神经元与上一个层所有神经元相连接，实现对前一层的线性组合， 前一层的输出乘以权重、相加然后输出**</font>\n",
    "## 输入和输出都是二维张量$[batch\\_size, size]$  \n",
    "<font color=\"red\">$torch.nn.Linear(in\\_features, out\\_features, bias=True)$</font>  \n",
    "**in_features**:&nbsp;&nbsp;输入的二维张量的大小，即输入的 <strong>[batch_size, size] </strong>  中的size  \n",
    "**out_features：**&nbsp;&nbsp;输出的二维张量的大小，即输出的二维张量的形状为  <strong>[batch_size，output_size]</strong> ，当然，它也代表了该全连接层的神经元个数。\n",
    "\n",
    "* 从输入输出的张量的shape角度来理解:\n",
    "    * 相当于一个输入为  <strong>[batch_size, in_features]</strong>  的张量变换成了  <strong>[batch_size, out_features]</strong>  的输出张量。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 12288])\n",
      "torch.Size([1, 10])\n"
     ]
    }
   ],
   "source": [
    "# in_features由输入张量的形状决定，out_features则决定了输出张量的形状 \n",
    "connected_layer = nn.Linear(in_features = 64 * 64 * 3, out_features = 10)\n",
    "# 实例化一个线性层， 也称为全连接层\n",
    "\n",
    "# 假定输入的图像形状为[64,64,3]\n",
    "input = torch.randn(1,64,64,3)# 一个batch一张图片\n",
    "\n",
    "# 将四维张量转换为二维张量之后，才能作为全连接层的输入\n",
    "input = input.view(1,64 * 64 * 3)\n",
    "print(input.shape)\n",
    "output = connected_layer(input) # 调用全连接层\n",
    "print(output.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.pad_sequence()填充句子长度\n",
    "<font size=4, color=\"red\">$$pad\\_sequence(sequences, batch\\_first=False, padding\\_value=0)$$</font>\n",
    "**在输入为一个batch的句子，输入的句子的长度必然是不同的，为了保证维度一致我们会把其中的短句子padding到和长句等长，但是我们又不希望这些padding的值参与训练，因此这里就是告诉RNN模型输入的padding情况。**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[10.,  3.,  4.,  5.],\n",
       "        [ 6.,  5.,  4., -1.],\n",
       "        [ 5.,  6.,  8., -1.]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_in = [torch.Tensor([10,3,4,5]),torch.Tensor([6,5,4]),torch.Tensor([5,6,8])]\n",
    "\n",
    "pad = nn.utils.rnn.pad_sequence(test_in, batch_first=True, padding_value=-1)\n",
    "pad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* pad_sequence()填充，[data和datasets库的应用](#data)\n",
    "    * shuffle数据后，先条一条的读取数据，当读满一个batch_size时,把数据送福collate_fn进一步处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1, 1, 1, 1, 1, 1, 1],\n",
      "        [2, 2, 2, 2, 2, 2, 0],\n",
      "        [3, 3, 3, 3, 3, 0, 0],\n",
      "        [4, 4, 4, 4, 0, 0, 0],\n",
      "        [5, 5, 5, 0, 0, 0, 0],\n",
      "        [6, 6, 0, 0, 0, 0, 0],\n",
      "        [7, 0, 0, 0, 0, 0, 0]])\n",
      "====================万能的分隔符====================\n",
      "====================我现在在Mydata里====================\n",
      "tensor([5, 5, 5])\n",
      "====================我现在在Mydata里====================\n",
      "tensor([2, 2, 2, 2, 2, 2])\n",
      "====================我现在在Mydata里====================\n",
      "tensor([7])\n",
      "collate_fn的batch：\n",
      " [tensor([5, 5, 5]), tensor([2, 2, 2, 2, 2, 2]), tensor([7])]\n",
      "pad之后的batch：\n",
      " tensor([[2, 2, 2, 2, 2, 2],\n",
      "        [5, 5, 5, 0, 0, 0],\n",
      "        [7, 0, 0, 0, 0, 0]])\n",
      "====================我现在在Mydata里====================\n",
      "tensor([6, 6])\n",
      "====================我现在在Mydata里====================\n",
      "tensor([4, 4, 4, 4])\n",
      "====================我现在在Mydata里====================\n",
      "tensor([1, 1, 1, 1, 1, 1, 1])\n",
      "collate_fn的batch：\n",
      " [tensor([6, 6]), tensor([4, 4, 4, 4]), tensor([1, 1, 1, 1, 1, 1, 1])]\n",
      "pad之后的batch：\n",
      " tensor([[1, 1, 1, 1, 1, 1, 1],\n",
      "        [4, 4, 4, 4, 0, 0, 0],\n",
      "        [6, 6, 0, 0, 0, 0, 0]])\n",
      "====================我现在在Mydata里====================\n",
      "tensor([3, 3, 3, 3, 3])\n",
      "collate_fn的batch：\n",
      " [tensor([3, 3, 3, 3, 3])]\n",
      "pad之后的batch：\n",
      " tensor([[3, 3, 3, 3, 3]])\n",
      "====================我现在在Mydata里====================\n",
      "tensor([3, 3, 3, 3, 3])\n",
      "====================我现在在Mydata里====================\n",
      "tensor([6, 6])\n",
      "====================我现在在Mydata里====================\n",
      "tensor([1, 1, 1, 1, 1, 1, 1])\n",
      "collate_fn的batch：\n",
      " [tensor([3, 3, 3, 3, 3]), tensor([6, 6]), tensor([1, 1, 1, 1, 1, 1, 1])]\n",
      "pad之后的batch：\n",
      " tensor([[1, 1, 1, 1, 1, 1, 1],\n",
      "        [3, 3, 3, 3, 3, 0, 0],\n",
      "        [6, 6, 0, 0, 0, 0, 0]])\n"
     ]
    }
   ],
   "source": [
    "import torch.utils.data as data\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "train_x = [torch.tensor([1, 1, 1, 1, 1, 1, 1]),\n",
    "           torch.tensor([2, 2, 2, 2, 2, 2]),\n",
    "           torch.tensor([3, 3, 3, 3, 3]),\n",
    "           torch.tensor([4, 4, 4, 4]),\n",
    "           torch.tensor([5, 5, 5]),\n",
    "           torch.tensor([6, 6]),\n",
    "           torch.tensor([7])]\n",
    "#  正常操作\n",
    "x = pad_sequence(train_x, batch_first=True)\n",
    "print(x)\n",
    "print(20*\"=\" + \"万能的分隔符\"+ 20*\"=\")\n",
    "\n",
    "class MyData(data.DataLoader):\n",
    "    def __init__(self, data_seq):\n",
    "        self.data_seq = data_seq\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.data_seq)\n",
    "    \n",
    "    def __getitem__(self, idx): \n",
    "        print(20*\"=\" + \"我现在在Mydata里\"+ 20*\"=\")\n",
    "        print(self.data_seq[idx])\n",
    "       \n",
    "        return self.data_seq[idx]\n",
    "\n",
    "def collate_fn(batch):\n",
    "    print(\"collate_fn的batch：\\n\", batch)\n",
    "    \n",
    "    batch.sort(key=lambda x: len(x), reverse=True)\n",
    "    batch = pad_sequence(batch, batch_first=True, padding_value=0)\n",
    "    print(\"pad之后的batch：\\n\",batch)\n",
    "    return batch\n",
    "    \n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # 实例化读取data的对象\n",
    "    data = MyData(train_x)\n",
    "    # 加载数据\n",
    "    data_loader = DataLoader(data, batch_size=3, shuffle=True, collate_fn=collate_fn)\n",
    "\n",
    "    for data in data_loader:\n",
    "        data\n",
    "\n",
    "    batch_x = iter(data_loader).next()\n",
    "    # next() 函数要和生成迭代器的 iter() 函数一起使用"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4-1.pack_padded_sequencepad过的句子进行压缩，然后送进网络进行训练,避免了无用的数据送入网络\n",
    "该操作是将pad过的句子进行压缩，然后送进网络进行训练，避免了无用的数据送入网络。\n",
    "\n",
    "pack_padded_sequence 有三个参数：input, lengths, batch_first 。\n",
    "\n",
    "* input 是上一步加过 padding 的数据，\n",
    "* lengths 是各个 sequence 的实际长度，\n",
    "* batch_first 是数据各个 dimension 按照 $[batch\\_size, sequence\\_length, data\\_dim]$顺序排列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pack: PackedSequence(data=tensor([1, 3, 6, 1, 3, 6, 1, 3, 0, 1, 3, 1, 3, 1, 1]), batch_sizes=tensor([3, 3, 3, 2, 2, 1, 1]), sorted_indices=None, unsorted_indices=None)\n"
     ]
    }
   ],
   "source": [
    "length = [7, 5, 3]\n",
    "pack= nn.utils.rnn.pack_padded_sequence(batch_x, length, batch_first=True)\n",
    "print('pack:',pack)# 数据以列的方式进行读取"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <span id=data>5.DataLoader()加载数据</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.weight decay权重衰减\n",
    "**$L_2$正则化：**让权重衰减到更小的值，在一定程度上**减少模型过拟合**的问题，所以**权重衰减也叫$L_2$正则化**。\n",
    "***\n",
    "### $L_2$正则化可以避免模型过拟合\n",
    "+  L2正则化项有让w变小的效果，但是为什么w变小可以防止过拟合呢？\n",
    "    1. 从模型的复杂度上解释\n",
    "        * 更小的权值w，从某种意义上说，表示网络的复杂度更低，对数据的拟合更好（这个法则也叫做奥卡姆剃刀）\n",
    "    2. 从数学方面的解释\n",
    "        * 过拟合的时候，拟合函数的系数往往非常大\n",
    "        * 过拟合就是**拟合函数需要顾忌每一个点，最终形成的拟合函数波动很大**。\n",
    "        * <font color=\"red\">在某些很小的区间里，函数值的变化很剧烈</font>\n",
    "            * 这就意味着函数在某些小区间里的**导数值（绝对值）非常大**，\n",
    "                * <font color=\"blue\">由于自变量值可大可小，所以只有系数足够大，才能保证导数值很大</font>。\n",
    "                * 正则化是通过约束参数的范数使其不要太大，所以可以在一定程度上减少过拟合情况。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed(1)\n",
    "\n",
    "n_hidden = 200 # 隐藏层神经单元数量\n",
    "max_iter = 2000 # 迭代次数\n",
    "disp_interval = 200 # 每200轮进行一次绘图显示\n",
    "lr_init = 0.01 #学习率\n",
    "\n",
    "\n",
    "# ============================ step 1/5 数据 ============================\n",
    "def gen_data(num_data=10, x_range=(-1, 1)):\n",
    "\n",
    "    w = 1.5 # 定义权重\n",
    "    train_x = torch.linspace(*x_range, num_data).unsqueeze_(1)# [10, 1]\n",
    "    #定义数据集X，*x_range表示将参数名为x_range的内容打散，即生成-1到1的10个数并扩展维度\n",
    "    train_y = w*train_x + torch.normal(0, 0.5, size=train_x.size())\n",
    "    \n",
    "    test_x = torch.linspace(*x_range, num_data).unsqueeze_(1)\n",
    "    test_y = w*test_x + torch.normal(0, 0.3, size=test_x.size())\n",
    "\n",
    "    return train_x, train_y, test_x, test_y\n",
    "\n",
    "\n",
    "train_x, train_y, test_x, test_y = gen_data()\n",
    "\n",
    "# train_x,test_y,test_x,test_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 2/5 模型 ============================\n",
    "class MLP(nn.Module):\n",
    "    def __init__(self, neural_num):\n",
    "        super(MLP, self).__init__()\n",
    "        self.linears = nn.Sequential(# 利用容器简历模型\n",
    "            nn.Linear(1, neural_num),\n",
    "            # [batch_size(每个输入的样本大小),size(输出样本的大小)]\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(neural_num, neural_num),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(neural_num, neural_num),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(neural_num, 1),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.linears(x)\n",
    "\n",
    "\n",
    "net_normal = MLP(neural_num=n_hidden)\n",
    "net_weight_decay = MLP(neural_num=n_hidden)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
